"""
ProphecyModel - AI量化预测模型
基于多因子和个股预测的智能投资组合管理系统
"""

import pandas as pd
import numpy as np
import akshare as ak
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis as QDA
from sklearn.svm import SVC
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
import pickle
import warnings
warnings.filterwarnings('ignore')

class ProphecyModel:
    """
    AI量化预测模型
    结合多因子预测（模型1）和个股预测（模型2）的投资组合管理系统
    """
    
    def __init__(self, config=None):
        """
        初始化模型
        
        Args:
            config (dict): 配置参数
        """
        self.config = self._default_config()
        if config:
            self.config.update(config)
        
        self.models = self._init_models()
        self.scaler = StandardScaler()
        self.feature_names = []
        self.trained_feature_count = 0  # 记录训练时的特征数量
        
        # 新增：股票筛选相关属性
        self.filtered_x_symbols = []  # 筛选后的X股票列表
        self.stock_quality_scores = {}  # 股票质量评分
        self.low_accuracy_stocks = []  # 低精度股票列表
        self.correlation_matrix = None  # 相关性矩阵
        self.feature_importance_ranking = {}  # 特征重要性排名
    
    def _default_config(self):
        """默认配置"""
        return {
            'lookback_days': 60,        # 回看天数
            'max_positions': 10,        # 最大持仓数量
            'max_position_size': 0.2,   # 单股最大仓位
            'min_correlation': 0.5,     # 最小相关性
            'train_test_split': 0.8,    # 训练集比例
            'test_days': 60,            # 测试集天数
            'rebalance_frequency': 5,   # 调仓频率（天）
            'commission': 0.001,        # 手续费率
            'slippage': 0.0005,         # 滑点
        }
    
    def _init_models(self):
        """初始化机器学习模型（添加正则化防止过拟合）"""
        return {
            'logistic': LogisticRegression(random_state=42, C=0.1, max_iter=1000),
            'lda': LDA(),
            'qda': QDA(),
            'svc': SVC(probability=True, random_state=42, C=0.1, kernel='rbf'),
            'rf': RandomForestClassifier(n_estimators=100, random_state=42, max_depth=10, min_samples_split=10, min_samples_leaf=5),
            'gb': GradientBoostingClassifier(random_state=42, n_estimators=100, max_depth=5, learning_rate=0.1, subsample=0.8)
        }
    
    def reset_model_state(self):
        """重置模型状态"""
        self.scaler = StandardScaler()
        self.feature_names = []
        self.trained_feature_count = 0
        
        # 重置筛选相关状态
        self.filtered_x_symbols = []
        self.stock_quality_scores = {}
        self.low_accuracy_stocks = []
        self.correlation_matrix = None
        self.feature_importance_ranking = {}
        
        print("✅ 模型状态已重置")
    
    def get_market_data(self, symbols, start_date=None, end_date=None):
        """
        获取真实市场数据
        
        Args:
            symbols (list): 股票代码列表
            start_date (str): 开始日期，格式：'2023-01-01'
            end_date (str): 结束日期，格式：'2024-01-01'
            
        Returns:
            dict: 股票数据字典
        """
        if start_date is None:
            start_date = '2023-01-01'
        if end_date is None:
            end_date = '2024-01-01'
            
        market_data = {}
        
        for symbol in symbols:
            try:
                print(f"📥 获取 {symbol} 数据...")
                
                # 使用 akshare 获取美股数据
                if symbol.endswith('.US'):
                    # 美股数据
                    stock_code = symbol.replace('.US', '')
                    print(f"   尝试获取美股: {stock_code}")
                    try:
                        df = ak.stock_us_daily(symbol=stock_code)
                        print(f"   美股数据形状: {df.shape if df is not None else 'None'}")
                        if df is not None and len(df) > 0:
                            df['symbol'] = symbol
                        else:
                            print(f"   ⚠️ 美股数据为空，尝试其他方法")
                            # 尝试使用其他方法获取美股数据
                            df = ak.stock_us_hist(symbol=stock_code, period="daily", 
                                                start_date=start_date, end_date=end_date)
                            if df is not None and len(df) > 0:
                                df['symbol'] = symbol
                    except Exception as e:
                        print(f"   ❌ 美股数据获取失败: {str(e)}")
                        df = None
                else:
                    # A股数据
                    print(f"   尝试获取A股: {symbol}")
                    try:
                        df = ak.stock_zh_a_hist(symbol=symbol, period="daily", 
                                              start_date=start_date, end_date=end_date)
                        print(f"   A股数据形状: {df.shape if df is not None else 'None'}")
                        if df is not None and len(df) > 0:
                            df['symbol'] = symbol
                    except Exception as e:
                        print(f"   ❌ A股数据获取失败: {str(e)}")
                        df = None
                
                # 统一列名
                df = self._standardize_columns(df)
                
                # 计算技术指标
                df = self._calculate_technical_indicators(df)
                
                market_data[symbol] = df
                print(f"✅ {symbol} 数据获取成功，共 {len(df)} 条记录")
                
            except Exception as e:
                print(f"❌ 获取 {symbol} 数据失败: {str(e)}")
                continue
        
        return market_data
    
    def _standardize_columns(self, df):
        """标准化数据列名"""
        if df is None or len(df) == 0:
            print("⚠️ 数据为空")
            return None
            
        print(f"📊 原始列名: {list(df.columns)}")
        
        # 美股数据列名映射
        us_column_mapping = {
            'Date': 'date',
            'Open': 'open',
            'High': 'high', 
            'Low': 'low',
            'Close': 'close',
            'Volume': 'volume',
            'Adj Close': 'adj_close'
        }
        
        # A股数据列名映射
        cn_column_mapping = {
            '日期': 'date',
            '开盘': 'open',
            '最高': 'high', 
            '最低': 'low',
            '收盘': 'close',
            '成交量': 'volume',
            '成交额': 'amount',
            '振幅': 'amplitude',
            '涨跌幅': 'pct_change',
            '涨跌额': 'change',
            '换手率': 'turnover'
        }
        
        # 尝试美股映射
        if any(col in df.columns for col in us_column_mapping.keys()):
            df = df.rename(columns=us_column_mapping)
            print("✅ 使用美股列名映射")
        # 尝试A股映射
        elif any(col in df.columns for col in cn_column_mapping.keys()):
            df = df.rename(columns=cn_column_mapping)
            print("✅ 使用A股列名映射")
        # 检查是否已经是标准格式
        elif all(col in df.columns for col in ['date', 'open', 'high', 'low', 'close', 'volume']):
            print("✅ 列名已经是标准格式")
        else:
            print(f"⚠️ 无法识别的列名格式: {list(df.columns)}")
            return None
        
        print(f"📊 映射后列名: {list(df.columns)}")
        
        # 确保必要的列存在
        required_columns = ['date', 'open', 'high', 'low', 'close', 'volume']
        missing_columns = [col for col in required_columns if col not in df.columns]
        if missing_columns:
            print(f"⚠️ 缺少必要列: {missing_columns}")
            return None
        
        # 转换日期格式
        df['date'] = pd.to_datetime(df['date'])
        df = df.sort_values('date').reset_index(drop=True)
        
        print(f"✅ 数据标准化完成，共 {len(df)} 条记录")
        return df
    
    def _calculate_technical_indicators(self, df):
        """计算技术指标"""
        try:
            # 基础价格数据
            df['pct_change'] = df['close'].pct_change()
            df['returns'] = df['pct_change'] * 100
            
            # 移动平均线
            df['ma5'] = df['close'].rolling(window=5).mean()
            df['ma10'] = df['close'].rolling(window=10).mean()
            df['ma20'] = df['close'].rolling(window=20).mean()
            
            # RSI
            delta = df['close'].diff()
            gain = (delta.where(delta > 0, 0)).rolling(window=14).mean()
            loss = (-delta.where(delta < 0, 0)).rolling(window=14).mean()
            # 避免除零错误
            rs = gain / loss.replace(0, 1e-10)
            df['rsi'] = 100 - (100 / (1 + rs))
            
            # 布林带
            df['bb_middle'] = df['close'].rolling(window=20).mean()
            bb_std = df['close'].rolling(window=20).std()
            df['bb_upper'] = df['bb_middle'] + (bb_std * 2)
            df['bb_lower'] = df['bb_middle'] - (bb_std * 2)
            
            # 成交量指标
            df['volume_ma'] = df['volume'].rolling(window=20).mean()
            # 避免除零错误
            df['volume_ratio'] = df['volume'] / df['volume_ma'].replace(0, 1e-10)
            
            # 波动率
            df['volatility'] = df['returns'].rolling(window=20).std()
            
            # MACD
            exp1 = df['close'].ewm(span=12).mean()
            exp2 = df['close'].ewm(span=26).mean()
            df['macd'] = exp1 - exp2
            df['macd_signal'] = df['macd'].ewm(span=9).mean()
            df['macd_histogram'] = df['macd'] - df['macd_signal']
            
            # 价格位置 - 避免除零错误
            price_range = df['high'] - df['low']
            df['price_position'] = (df['close'] - df['low']) / price_range.replace(0, 1e-10)
            
            # 滞后特征
            for lag in [1, 2, 3, 5]:
                df[f'returns_lag_{lag}'] = df['returns'].shift(lag)
                df[f'volume_lag_{lag}'] = df['volume'].shift(lag)
            
            # 清理无穷大和NaN值
            df = df.replace([np.inf, -np.inf], np.nan)
            df = df.fillna(0)
            
        except Exception as e:
            print(f"❌ 计算技术指标失败: {str(e)}")
        
        return df
    
    def prepare_features(self, market_data, x_symbols, y_symbols):
        """
        准备特征数据，实现训练集和测试集分离（4:1比例）
        
        Args:
            market_data (dict): 市场数据
            x_symbols (list): 因子股票列表
            y_symbols (list): 目标股票列表
            
        Returns:
            tuple: (X_train, X_test, y_train, y_test, feature_names)
        """
        print("🔧 准备特征数据...")
        
        # 保存数据供后续筛选分析使用
        self.market_data = market_data
        self.x_symbols = x_symbols
        self.y_symbols = y_symbols
        
        # 合并所有数据
        all_data = {}
        for symbol in x_symbols + y_symbols:
            if symbol in market_data and market_data[symbol] is not None:
                all_data[symbol] = market_data[symbol]
        
        if not all_data:
            print("❌ 没有可用的市场数据")
            return None, None, None, None
        
        # 找到共同的时间范围
        min_length = min(len(df) for df in all_data.values())
        print(f"📊 数据长度: {min_length}")
        
        if min_length < 50:
            print("❌ 数据长度不足，至少需要50条记录")
            return None, None, None, None
        
        # 创建特征矩阵
        features = []
        targets = []
        dates = []
        
        # 使用最后 min_length 天的数据
        for i in range(min_length):
            feature_row = []
            target_row = []
            
            # X特征（因子股票）- 确保顺序一致
            for symbol in x_symbols:
                if symbol in all_data:
                    df = all_data[symbol]
                    idx = len(df) - min_length + i
                    if idx >= 0:
                        # 添加技术指标作为特征（固定顺序）
                        feature_row.extend([
                            df.iloc[idx]['returns'] if not pd.isna(df.iloc[idx]['returns']) else 0,
                            df.iloc[idx]['rsi'] if not pd.isna(df.iloc[idx]['rsi']) else 50,
                            df.iloc[idx]['volume_ratio'] if not pd.isna(df.iloc[idx]['volume_ratio']) else 1,
                            df.iloc[idx]['volatility'] if not pd.isna(df.iloc[idx]['volatility']) else 0,
                            df.iloc[idx]['macd'] if not pd.isna(df.iloc[idx]['macd']) else 0,
                            df.iloc[idx]['price_position'] if not pd.isna(df.iloc[idx]['price_position']) else 0.5
                        ])
                    else:
                        feature_row.extend([0, 50, 1, 0, 0, 0.5])
                else:
                    feature_row.extend([0, 50, 1, 0, 0, 0.5])
            
            # Y目标（目标股票）
            for symbol in y_symbols:
                if symbol in all_data:
                    df = all_data[symbol]
                    idx = len(df) - min_length + i
                    if idx >= 0:
                        # 使用次日涨跌作为目标（1为上涨，0为下跌）
                        if idx + 1 < len(df):
                            next_return = df.iloc[idx + 1]['returns'] if not pd.isna(df.iloc[idx + 1]['returns']) else 0
                            target_row.append(1 if next_return > 0 else 0)
                        else:
                            target_row.append(0)
                    else:
                        target_row.append(0)
                else:
                    target_row.append(0)
            
            if len(feature_row) > 0 and len(target_row) > 0:
                features.append(feature_row)
                targets.append(target_row)
                dates.append(all_data[x_symbols[0]].iloc[len(all_data[x_symbols[0]]) - min_length + i]['date'])
        
        # 转换为DataFrame
        feature_names = []
        for symbol in x_symbols:
            feature_names.extend([
                f'{symbol}_returns', f'{symbol}_rsi', f'{symbol}_volume_ratio',
                f'{symbol}_volatility', f'{symbol}_macd', f'{symbol}_price_position'
            ])
        
        X = pd.DataFrame(features, columns=feature_names)
        y = pd.DataFrame(targets, columns=y_symbols)
        
        # 记录特征数量
        self.feature_names = feature_names
        self.trained_feature_count = len(feature_names)
        
        # 训练集和测试集分离（4:1比例）
        split_idx = int(len(X) * self.config['train_test_split'])
        
        X_train = X.iloc[:split_idx]
        X_test = X.iloc[split_idx:]
        y_train = y.iloc[:split_idx]
        y_test = y.iloc[split_idx:]
        
        print(f"✅ 特征数量: {len(feature_names)}")
        print(f"✅ 训练集: {len(X_train)} 条")
        print(f"✅ 测试集: {len(X_test)} 条")
        print(f"✅ 目标数量: {len(y_symbols)}")
        
        return X_train, X_test, y_train, y_test
    
    def analyze_stock_correlations(self, market_data, x_symbols, y_symbols):
        """
        分析股票相关性，识别冗余和低相关性股票
        
        Args:
            market_data (dict): 市场数据
            x_symbols (list): 因子股票列表
            y_symbols (list): 目标股票列表
            
        Returns:
            dict: 相关性分析结果
        """
        print("🔍 分析股票相关性...")
        
        # 计算收益率相关性矩阵
        returns_data = {}
        for symbol in x_symbols + y_symbols:
            if symbol in market_data and market_data[symbol] is not None:
                df = market_data[symbol]
                if 'returns' in df.columns:
                    returns_data[symbol] = df['returns'].dropna()
        
        if len(returns_data) < 2:
            print("❌ 数据不足，无法计算相关性")
            return {}
        
        # 创建收益率DataFrame
        returns_df = pd.DataFrame(returns_data)
        returns_df = returns_df.fillna(method='ffill').fillna(0)
        
        # 计算相关性矩阵
        correlation_matrix = returns_df.corr()
        self.correlation_matrix = correlation_matrix
        
        # 分析X股票与Y股票的相关性
        x_y_correlations = {}
        for x_symbol in x_symbols:
            if x_symbol in correlation_matrix.index:
                correlations = []
                for y_symbol in y_symbols:
                    if y_symbol in correlation_matrix.columns:
                        corr = correlation_matrix.loc[x_symbol, y_symbol]
                        correlations.append(abs(corr))
                
                if correlations:
                    # 计算平均相关性
                    avg_correlation = np.mean(correlations)
                    x_y_correlations[x_symbol] = {
                        'avg_correlation': avg_correlation,
                        'max_correlation': max(correlations),
                        'min_correlation': min(correlations),
                        'correlation_std': np.std(correlations)
                    }
        
        # 分析X股票之间的相关性（识别冗余）
        x_x_correlations = {}
        for i, x1 in enumerate(x_symbols):
            if x1 in correlation_matrix.index:
                correlations_with_others = []
                for j, x2 in enumerate(x_symbols):
                    if i != j and x2 in correlation_matrix.index:
                        corr = abs(correlation_matrix.loc[x1, x2])
                        correlations_with_others.append(corr)
                
                if correlations_with_others:
                    x_x_correlations[x1] = {
                        'avg_correlation_with_x': np.mean(correlations_with_others),
                        'max_correlation_with_x': max(correlations_with_others),
                        'redundancy_score': np.mean(correlations_with_others)  # 冗余度评分
                    }
        
        print("✅ 相关性分析完成")
        return {
            'x_y_correlations': x_y_correlations,
            'x_x_correlations': x_x_correlations,
            'correlation_matrix': correlation_matrix
        }
    
    def evaluate_stock_importance(self, model1_results, feature_names):
        """
        评估股票重要性，基于特征重要性排名
        
        Args:
            model1_results (dict): 模型1训练结果
            feature_names (list): 特征名称列表
            
        Returns:
            dict: 股票重要性评估结果
        """
        print("🔍 评估股票重要性...")
        
        stock_importance = {}
        
        for target_symbol, result in model1_results.items():
            if 'feature_importance' in result:
                feature_importance = result['feature_importance']
                
                # 处理numpy数组格式的特征重要性
                if isinstance(feature_importance, np.ndarray):
                    # 将numpy数组转换为字典，映射特征名称到重要性值
                    feature_importance_dict = dict(zip(feature_names, feature_importance))
                else:
                    feature_importance_dict = feature_importance
                
                # 按股票分组特征重要性
                for feature_name, importance in feature_importance_dict.items():
                    # 提取股票代码（特征名称格式：AAPL.US_returns, AAPL.US_rsi等）
                    stock_code = feature_name.split('_')[0]
                    
                    if stock_code not in stock_importance:
                        stock_importance[stock_code] = {
                            'total_importance': 0,
                            'feature_count': 0,
                            'features': {}
                        }
                    
                    stock_importance[stock_code]['total_importance'] += importance
                    stock_importance[stock_code]['feature_count'] += 1
                    stock_importance[stock_code]['features'][feature_name] = importance
        
        # 计算平均重要性并排序
        for stock_code in stock_importance:
            stock_importance[stock_code]['avg_importance'] = (
                stock_importance[stock_code]['total_importance'] / 
                stock_importance[stock_code]['feature_count']
            )
        
        # 按平均重要性排序
        sorted_stocks = sorted(
            stock_importance.items(),
            key=lambda x: x[1]['avg_importance'],
            reverse=True
        )
        
        self.feature_importance_ranking = dict(sorted_stocks)
        
        print("✅ 股票重要性评估完成")
        return stock_importance
    
    def calculate_stock_quality_scores(self, correlation_analysis, importance_analysis, model1_results):
        """
        计算股票质量评分，综合考虑相关性、重要性和模型精度
        
        Args:
            correlation_analysis (dict): 相关性分析结果
            importance_analysis (dict): 重要性分析结果
            model1_results (dict): 模型1训练结果
            
        Returns:
            dict: 股票质量评分
        """
        print("🔍 计算股票质量评分...")
        
        quality_scores = {}
        x_symbols = list(importance_analysis.keys())
        
        for symbol in x_symbols:
            score = 0
            factors = {}
            
            # 1. 相关性评分 (30%)
            if 'x_y_correlations' in correlation_analysis and symbol in correlation_analysis['x_y_correlations']:
                x_y_corr = correlation_analysis['x_y_correlations'][symbol]
                correlation_score = x_y_corr['avg_correlation'] * 30
                score += correlation_score
                factors['correlation_score'] = correlation_score
            else:
                factors['correlation_score'] = 0
            
            # 2. 重要性评分 (40%)
            if symbol in importance_analysis:
                importance_score = importance_analysis[symbol]['avg_importance'] * 40
                score += importance_score
                factors['importance_score'] = importance_score
            else:
                factors['importance_score'] = 0
            
            # 3. 冗余度评分 (20%) - 冗余度越低越好
            if 'x_x_correlations' in correlation_analysis and symbol in correlation_analysis['x_x_correlations']:
                x_x_corr = correlation_analysis['x_x_correlations'][symbol]
                redundancy_score = (1 - x_x_corr['redundancy_score']) * 20
                score += redundancy_score
                factors['redundancy_score'] = redundancy_score
            else:
                factors['redundancy_score'] = 0
            
            # 4. 模型精度评分 (10%) - 基于模型1的预测精度
            model_accuracy_score = 0
            if symbol in model1_results:
                model_accuracy_score = model1_results[symbol].get('accuracy', 0.5) * 10
                score += model_accuracy_score
            factors['model_accuracy_score'] = model_accuracy_score
            
            # 计算总评分
            factors['total_score'] = score
            quality_scores[symbol] = factors
        
        # 按总评分排序
        sorted_quality = sorted(
            quality_scores.items(),
            key=lambda x: x[1]['total_score'],
            reverse=True
        )
        
        self.stock_quality_scores = dict(sorted_quality)
        
        print("✅ 股票质量评分计算完成")
        return quality_scores
    
    def filter_stocks(self, quality_scores, min_quality_threshold=60, max_stocks=15):
        """
        基于质量评分筛选股票
        
        Args:
            quality_scores (dict): 股票质量评分
            min_quality_threshold (float): 最小质量阈值
            max_stocks (int): 最大股票数量
            
        Returns:
            list: 筛选后的股票列表
        """
        print("🔍 筛选股票...")
        
        # 按质量评分筛选
        qualified_stocks = []
        for symbol, scores in quality_scores.items():
            if scores['total_score'] >= min_quality_threshold:
                qualified_stocks.append((symbol, scores['total_score']))
        
        # 按评分排序并限制数量
        qualified_stocks.sort(key=lambda x: x[1], reverse=True)
        filtered_stocks = [stock[0] for stock in qualified_stocks[:max_stocks]]
        
        self.filtered_x_symbols = filtered_stocks
        
        print(f"✅ 股票筛选完成: {len(filtered_stocks)}/{len(quality_scores)} 只股票通过筛选")
        print(f"   筛选阈值: {min_quality_threshold}")
        print(f"   最大数量: {max_stocks}")
        
        return filtered_stocks
    
    def identify_low_accuracy_stocks(self, model1_results, accuracy_threshold=0.6):
        """
        识别模型精度较低的股票
        
        Args:
            model1_results (dict): 模型1训练结果
            accuracy_threshold (float): 精度阈值
            
        Returns:
            list: 低精度股票列表
        """
        print("🔍 识别低精度股票...")
        
        low_accuracy_stocks = []
        
        for symbol, result in model1_results.items():
            if 'accuracy' in result:
                accuracy = result['accuracy']
                if accuracy < accuracy_threshold:
                    low_accuracy_stocks.append({
                        'symbol': symbol,
                        'accuracy': accuracy,
                        'threshold': accuracy_threshold
                    })
        
        # 按精度排序
        low_accuracy_stocks.sort(key=lambda x: x['accuracy'])
        self.low_accuracy_stocks = low_accuracy_stocks
        
        if low_accuracy_stocks:
            print(f"⚠️ 发现 {len(low_accuracy_stocks)} 只低精度股票:")
            for stock in low_accuracy_stocks:
                print(f"   {stock['symbol']}: 精度 {stock['accuracy']:.3f} < {stock['threshold']}")
        else:
            print("✅ 所有股票精度都达到要求")
        
        return low_accuracy_stocks
    
    def get_filtered_stock_recommendations(self):
        """
        获取股票筛选建议
        
        Returns:
            dict: 筛选建议
        """
        recommendations = {
            'filtered_stocks': self.filtered_x_symbols,
            'low_accuracy_stocks': self.low_accuracy_stocks,
            'quality_ranking': list(self.stock_quality_scores.keys()),
            'summary': {
                'total_analyzed': len(self.stock_quality_scores),
                'filtered_count': len(self.filtered_x_symbols),
                'low_accuracy_count': len(self.low_accuracy_stocks)
            }
        }
        
        return recommendations
    
    def print_stock_filtering_report(self):
        """
        打印股票筛选报告
        """
        print("\n" + "="*60)
        print("📊 股票筛选报告")
        print("="*60)
        
        # 筛选结果
        print(f"\n🎯 筛选结果:")
        print(f"   总分析股票: {len(self.stock_quality_scores)}")
        print(f"   通过筛选: {len(self.filtered_x_symbols)}")
        print(f"   低精度股票: {len(self.low_accuracy_stocks)}")
        
        # 筛选后的股票列表
        if self.filtered_x_symbols:
            print(f"\n✅ 筛选后的股票列表 (按质量评分排序):")
            for i, symbol in enumerate(self.filtered_x_symbols[:10], 1):  # 显示前10只
                if symbol in self.stock_quality_scores:
                    score = self.stock_quality_scores[symbol]['total_score']
                    print(f"   {i:2d}. {symbol}: {score:.2f}")
                if i >= 10:
                    print(f"   ... 还有 {len(self.filtered_x_symbols) - 10} 只股票")
                    break
        
        # 低精度股票
        if self.low_accuracy_stocks:
            print(f"\n⚠️ 低精度股票 (建议在模型2和回测中排除):")
            for stock in self.low_accuracy_stocks:
                print(f"   {stock['symbol']}: 精度 {stock['accuracy']:.3f} < {stock['threshold']}")
        
        # 质量评分详情
        print(f"\n📈 股票质量评分详情:")
        for i, (symbol, scores) in enumerate(list(self.stock_quality_scores.items())[:5], 1):
            print(f"   {i}. {symbol}:")
            print(f"      相关性: {scores['correlation_score']:.2f}")
            print(f"      重要性: {scores['importance_score']:.2f}")
            print(f"      冗余度: {scores['redundancy_score']:.2f}")
            print(f"      模型精度: {scores['model_accuracy_score']:.2f}")
            print(f"      总分: {scores['total_score']:.2f}")
        
        print("="*60)
    
    def _detect_overfitting(self, train_accuracy, test_accuracy, threshold=0.1):
        """检测过拟合"""
        if train_accuracy - test_accuracy > threshold:
            return True, train_accuracy - test_accuracy
        return False, 0
    
    def _get_overfitting_recommendations(self, overfitting_detected, accuracy_gap):
        """获取过拟合改进建议"""
        if not overfitting_detected:
            return ["✅ 模型表现良好，无明显过拟合"]
        
        recommendations = [
            f"⚠️ 检测到过拟合，训练准确率比测试准确率高 {accuracy_gap:.3f}",
            "💡 改进建议:",
            "   1. 增加正则化参数",
            "   2. 减少模型复杂度",
            "   3. 增加训练数据",
            "   4. 使用交叉验证",
            "   5. 特征选择或降维"
        ]
        return recommendations
    
    def train_model_1(self, X_train, y_train, model_name='rf', auto_filter=True, enable_advanced_optimization=True):
        """
        训练模型1：多因子预测模型
        
        Args:
            X_train (DataFrame): 训练特征
            y_train (DataFrame): 训练目标
            model_name (str): 模型名称
            auto_filter (bool): 是否自动进行基础股票筛选分析
            enable_advanced_optimization (bool): 是否启用高级特征选择和优化
            
        Returns:
            dict: 训练结果
        """
        print(f"🚀 开始训练模型1：多因子预测模型 ({model_name})")
        
        # 验证特征数量
        if self.trained_feature_count > 0 and len(X_train.columns) != self.trained_feature_count:
            print(f"⚠️ 训练特征数量不匹配: 期望 {self.trained_feature_count}, 实际 {len(X_train.columns)}")
            return {}
        
        results = {}
        model = self.models.get(model_name, self.models['rf'])
        
        for i, target_col in enumerate(y_train.columns):
            print(f"   训练目标 {i+1}: {target_col}")
            
            # 准备数据
            y_target = y_train[target_col].values
            
            # 标准化特征
            X_scaled = self.scaler.fit_transform(X_train)
            
            # 训练模型
            model.fit(X_scaled, y_target)
            
            # 预测
            y_pred = model.predict(X_scaled)
            accuracy = accuracy_score(y_target, y_pred)
            
            results[target_col] = {
                'model': model,
                'accuracy': accuracy,
                'predictions': y_pred,
                'feature_importance': self._get_feature_importance(model, model_name)
            }
            
            print(f"   ✅ {target_col}: 准确率 {accuracy:.3f}")
        
        # 高级特征选择和模型优化流程
        if enable_advanced_optimization and hasattr(self, 'market_data') and hasattr(self, 'x_symbols') and hasattr(self, 'y_symbols'):
            print("\n🔍 开始高级特征选择和模型优化流程...")
            try:
                # 执行高级特征选择和优化
                optimization_result = self.advanced_feature_selection_and_optimization(
                    results, self.market_data, self.x_symbols, self.y_symbols
                )
                
                if optimization_result:
                    # 打印优化报告
                    self.print_optimization_report()
                    
                    # 更新模型状态，使用优化后的股票列表
                    optimized_stocks = optimization_result['optimized_stocks']
                    print(f"\n🎯 模型1训练完成，已生成优化后的股票池")
                    print(f"   原始股票: {len(self.x_symbols)} 只")
                    print(f"   优化后股票: {len(optimized_stocks)} 只")
                    print(f"   筛选比例: {len(optimized_stocks)/len(self.x_symbols)*100:.1f}%")
                    
                    # 保存优化结果供后续使用
                    self.optimization_result = optimization_result
                    
                print("✅ 高级特征选择和模型优化流程完成")
                
            except Exception as e:
                print(f"⚠️ 高级特征选择和模型优化流程失败: {e}")
        
        # 基础股票筛选分析（保持向后兼容）
        elif auto_filter and hasattr(self, 'market_data') and hasattr(self, 'x_symbols') and hasattr(self, 'y_symbols'):
            print("\n🔍 开始基础股票筛选分析...")
            try:
                # 1. 相关性分析
                correlation_analysis = self.analyze_stock_correlations(
                    self.market_data, self.x_symbols, self.y_symbols
                )
                
                # 2. 重要性评估
                importance_analysis = self.evaluate_stock_importance(results, X_train.columns)
                
                # 3. 质量评分计算
                quality_scores = self.calculate_stock_quality_scores(
                    correlation_analysis, importance_analysis, results
                )
                
                # 4. 股票筛选
                filtered_stocks = self.filter_stocks(quality_scores)
                
                # 5. 识别低精度股票
                low_accuracy_stocks = self.identify_low_accuracy_stocks(results)
                
                # 6. 打印筛选报告
                self.print_stock_filtering_report()
                
                print("✅ 基础股票筛选分析完成")
                
            except Exception as e:
                print(f"⚠️ 基础股票筛选分析失败: {e}")
        
        return results
    
    def train_model_2(self, market_data, symbol, model_name='rf'):
        """
        训练模型2：个股预测模型
        
        Args:
            market_data (dict): 市场数据
            symbol (str): 股票代码
            model_name (str): 模型名称
            
        Returns:
            dict: 训练结果
        """
        print(f"🚀 开始训练模型2：{symbol} 个股预测模型")
        
        if symbol not in market_data:
            print(f"❌ {symbol} 不在市场数据中")
            return None
        
        df = market_data[symbol]
        
        # 准备特征
        features = []
        targets = []
        
        for i in range(20, len(df) - 1):  # 使用前20天数据预测下一天
            feature_row = []
            
            # 技术指标特征
            for lag in [1, 2, 3, 5, 10]:
                feature_row.extend([
                    df.iloc[i-lag]['returns'] if not pd.isna(df.iloc[i-lag]['returns']) else 0,
                    df.iloc[i-lag]['rsi'] if not pd.isna(df.iloc[i-lag]['rsi']) else 50,
                    df.iloc[i-lag]['volume_ratio'] if not pd.isna(df.iloc[i-lag]['volume_ratio']) else 1,
                    df.iloc[i-lag]['volatility'] if not pd.isna(df.iloc[i-lag]['volatility']) else 0,
                    df.iloc[i-lag]['macd'] if not pd.isna(df.iloc[i-lag]['macd']) else 0
                ])
            
            # 目标：次日涨跌
            next_return = df.iloc[i+1]['returns'] if not pd.isna(df.iloc[i+1]['returns']) else 0
            target = 1 if next_return > 0 else 0
            
            features.append(feature_row)
            targets.append(target)
        
        if len(features) < 50:  # 数据不足
            print(f"❌ {symbol} 数据不足，至少需要50条记录")
            return None
        
        # 转换为DataFrame
        feature_names = []
        for lag in [1, 2, 3, 5, 10]:
            feature_names.extend([
                f'returns_lag_{lag}', f'rsi_lag_{lag}', f'volume_ratio_lag_{lag}',
                f'volatility_lag_{lag}', f'macd_lag_{lag}'
            ])
        
        X = pd.DataFrame(features, columns=feature_names)
        y = pd.Series(targets)
        
        # 训练集和测试集分离
        split_idx = int(len(X) * self.config['train_test_split'])
        X_train = X.iloc[:split_idx]
        X_test = X.iloc[split_idx:]
        y_train = y.iloc[:split_idx]
        y_test = y.iloc[split_idx:]
        
        # 训练模型
        model = self.models.get(model_name, self.models['rf'])
        X_scaled = self.scaler.fit_transform(X_train)
        model.fit(X_scaled, y_train)
        
        # 评估
        y_pred = model.predict(X_scaled)
        accuracy = accuracy_score(y_train, y_pred)
        
        result = {
            'model': model,
            'accuracy': accuracy,
            'feature_importance': self._get_feature_importance(model, model_name),
            'X_test': X_test,
            'y_test': y_test
        }
        
        print(f"   ✅ {symbol} 训练完成，准确率: {accuracy:.3f}")
        return result
    
    def _get_feature_importance(self, model, model_name):
        """获取特征重要性"""
        try:
            if hasattr(model, 'feature_importances_'):
                return model.feature_importances_
            elif hasattr(model, 'coef_'):
                return np.abs(model.coef_[0])
            else:
                return np.ones(len(self.feature_names)) / len(self.feature_names)
        except:
            return np.ones(len(self.feature_names)) / len(self.feature_names)
    
    def calculate_covariance_matrix(self, market_data, symbols):
        """计算协方差矩阵"""
        try:
            # 提取收益率数据
            returns_data = {}
            for symbol in symbols:
                if symbol in market_data:
                    df = market_data[symbol]
                    returns = df['returns'].dropna()
                    if len(returns) > 0:
                        returns_data[symbol] = returns
            
            if len(returns_data) < 2:
                return None
            
            # 对齐数据
            aligned_returns = pd.DataFrame(returns_data)
            aligned_returns = aligned_returns.dropna()
            
            if len(aligned_returns) < 30:  # 数据不足
                return None
            
            # 计算协方差矩阵
            cov_matrix = aligned_returns.cov()
            
            return cov_matrix
            
        except Exception as e:
            print(f"❌ 计算协方差矩阵失败: {str(e)}")
            return None
    
    def _validate_features(self, features, expected_count=None):
        """验证特征一致性"""
        if expected_count is None:
            expected_count = self.trained_feature_count
        
        if expected_count > 0 and len(features) != expected_count:
            print(f"❌ 特征数量不匹配: 期望 {expected_count}, 实际 {len(features)}")
            return False
        return True
    
    def generate_portfolio_recommendations(self, model1_results, model2_results, 
                                         market_data, x_symbols, y_symbols, 
                                         current_market_state='neutral', use_optimized_stocks=True):
        """
        生成投资组合建议
        
        Args:
            model1_results (dict): 模型1结果
            model2_results (dict): 模型2结果
            market_data (dict): 市场数据
            x_symbols (list): 因子股票列表
            y_symbols (list): 目标股票列表
            current_market_state (str): 当前市场状态
            use_optimized_stocks (bool): 是否使用优化后的股票列表
            
        Returns:
            dict: 投资组合建议
        """
        print("🎯 生成投资组合建议...")
        
        # 优先使用高级优化后的股票列表
        if use_optimized_stocks and hasattr(self, 'optimized_x_symbols') and self.optimized_x_symbols:
            effective_x_symbols = self.optimized_x_symbols
            print(f"✅ 使用高级优化后的股票列表: {len(effective_x_symbols)} 只股票")
            print(f"   优化后股票: {', '.join(effective_x_symbols[:5])}{'...' if len(effective_x_symbols) > 5 else ''}")
            optimization_source = "高级特征选择优化"
        # 其次使用基础筛选后的股票列表
        elif use_optimized_stocks and hasattr(self, 'filtered_x_symbols') and self.filtered_x_symbols:
            effective_x_symbols = self.filtered_x_symbols
            print(f"✅ 使用基础筛选后的股票列表: {len(effective_x_symbols)} 只股票")
            print(f"   筛选后股票: {', '.join(effective_x_symbols[:5])}{'...' if len(effective_x_symbols) > 5 else ''}")
            optimization_source = "基础股票筛选"
        else:
            effective_x_symbols = x_symbols
            print(f"ℹ️ 使用原始股票列表: {len(effective_x_symbols)} 只股票")
            optimization_source = "原始股票列表"
        
        # 排除低精度股票
        if hasattr(self, 'low_accuracy_stocks') and self.low_accuracy_stocks:
            low_accuracy_symbols = [stock['symbol'] for stock in self.low_accuracy_stocks]
            effective_x_symbols = [s for s in effective_x_symbols if s not in low_accuracy_symbols]
            print(f"⚠️ 排除低精度股票后: {len(effective_x_symbols)} 只股票")
        
        if not effective_x_symbols:
            print("❌ 没有可用的股票进行投资组合建议")
            return self._empty_recommendations()
        
        # 计算协方差矩阵（使用优化后的股票）
        cov_matrix = self.calculate_covariance_matrix(market_data, effective_x_symbols)
        
        # 获取最新特征
        latest_features = {}
        for symbol in effective_x_symbols:
            try:
                features = self._get_latest_features(market_data, effective_x_symbols, symbol)
                if features is not None:
                    latest_features[symbol] = features
            except Exception as e:
                print(f"⚠️ 获取 {symbol} 特征失败: {e}")
                continue
        
        if not latest_features:
            print("❌ 无法获取股票特征")
            return self._empty_recommendations()
        
        # 模型1预测（多因子预测）
        model1_predictions = {}
        for symbol in effective_x_symbols:
            if symbol in latest_features:
                try:
                    # 验证特征数量
                    features = latest_features[symbol]
                    if not self._validate_features(features):
                        continue
                    
                    # 标准化特征
                    features_scaled = self.scaler.transform([features])
                    
                    # 使用模型1进行预测
                    symbol_predictions = []
                    for target_symbol, result in model1_results.items():
                        if 'model' in result:
                            model = result['model']
                            pred = model.predict(features_scaled)[0]
                            prob = model.predict_proba(features_scaled)[0]
                            confidence = max(prob)
                            
                            symbol_predictions.append({
                                'target': target_symbol,
                                'prediction': 'rise' if pred == 1 else 'fall',
                                'confidence': confidence
                            })
                    
                    if symbol_predictions:
                        # 计算综合预测分数
                        rise_score = sum(p['confidence'] for p in symbol_predictions if p['prediction'] == 'rise')
                        fall_score = sum(p['confidence'] for p in symbol_predictions if p['prediction'] == 'fall')
                        
                        model1_predictions[symbol] = {
                            'predictions': symbol_predictions,
                            'rise_score': rise_score,
                            'fall_score': fall_score,
                            'net_score': rise_score - fall_score,
                            'overall_prediction': 'rise' if rise_score > fall_score else 'fall'
                        }
                        
                except Exception as e:
                    print(f"⚠️ 模型1预测 {symbol} 失败: {e}")
                    continue
        
        # 模型2预测（个股预测）
        model2_predictions = {}
        for symbol in effective_x_symbols:
            if symbol in model2_results:
                try:
                    result = model2_results[symbol]
                    if 'model' in result and 'accuracy' in result:
                        model2_predictions[symbol] = {
                            'prediction': result.get('prediction', 'neutral'),
                            'confidence': result.get('confidence', 0.5),
                            'accuracy': result.get('accuracy', 0.5)
                        }
                except Exception as e:
                    print(f"⚠️ 获取模型2预测 {symbol} 失败: {e}")
                    continue
        
        # 生成投资建议
        recommendations = {
            'portfolio': {},
            'market_state': current_market_state,
            'total_allocation': 0,
            'risk_metrics': {},
            'optimization_info': {
                'used_optimized_stocks': use_optimized_stocks,
                'optimization_source': optimization_source,
                'effective_stocks_count': len(effective_x_symbols),
                'original_stocks_count': len(x_symbols),
                'optimization_ratio': len(effective_x_symbols) / len(x_symbols) * 100 if len(x_symbols) > 0 else 0
            }
        }
        
        # 计算仓位权重
        total_score = 0
        position_scores = {}
        
        for symbol in effective_x_symbols:
            score = 0
            
            # 模型1分数
            if symbol in model1_predictions:
                model1_score = model1_predictions[symbol]['net_score']
                score += model1_score * 0.6  # 模型1权重60%
            
            # 模型2分数
            if symbol in model2_predictions:
                model2_score = model2_predictions[symbol]['confidence']
                if model2_predictions[symbol]['prediction'] == 'rise':
                    score += model2_score * 0.4  # 模型2权重40%
                else:
                    score -= model2_score * 0.4
            
            # 质量评分加成（如果使用高级优化）
            if hasattr(self, 'optimization_result') and self.optimization_result:
                feature_importance_analysis = self.optimization_result.get('feature_importance_analysis', {})
                if symbol in feature_importance_analysis:
                    importance_score = feature_importance_analysis[symbol]['avg_importance']
                    score += importance_score * 0.2  # 特征重要性权重20%
            # 基础质量评分加成（向后兼容）
            elif hasattr(self, 'stock_quality_scores') and symbol in self.stock_quality_scores:
                quality_bonus = self.stock_quality_scores[symbol]['total_score'] / 100
                score += quality_bonus * 0.2  # 质量评分权重20%
            
            position_scores[symbol] = score
            total_score += abs(score)
        
        # 分配仓位
        if total_score > 0:
            for symbol, score in position_scores.items():
                if abs(score) > 0.01:  # 最小阈值
                    weight = abs(score) / total_score
                    weight = min(weight, self.config['max_position_size'])  # 限制单股最大仓位
                    
                    recommendations['portfolio'][symbol] = {
                        'weight': weight,
                        'score': score,
                        'model1_prediction': model1_predictions.get(symbol, {}),
                        'model2_prediction': model2_predictions.get(symbol, {}),
                        'optimization_score': self._get_optimization_score(symbol)
                    }
        
        # 计算总配置
        recommendations['total_allocation'] = sum(pos['weight'] for pos in recommendations['portfolio'].values())
        
        # 计算风险指标
        if recommendations['portfolio']:
            recommendations['risk_metrics'] = self._calculate_risk_metrics(
                recommendations['portfolio'], cov_matrix
            )
        
        print(f"✅ 投资组合建议生成完成")
        print(f"   优化来源: {optimization_source}")
        print(f"   总配置: {recommendations['total_allocation']:.2%}")
        print(f"   股票数量: {len(recommendations['portfolio'])}")
        print(f"   优化比例: {recommendations['optimization_info']['optimization_ratio']:.1f}%")
        
        return recommendations
    
    def _get_optimization_score(self, symbol):
        """
        获取股票的优化评分
        
        Args:
            symbol (str): 股票代码
            
        Returns:
            float: 优化评分
        """
        if hasattr(self, 'optimization_result') and self.optimization_result:
            feature_importance_analysis = self.optimization_result.get('feature_importance_analysis', {})
            if symbol in feature_importance_analysis:
                return feature_importance_analysis[symbol]['avg_importance']
        
        # 向后兼容
        if hasattr(self, 'stock_quality_scores') and symbol in self.stock_quality_scores:
            return self.stock_quality_scores[symbol]['total_score'] / 100
        
        return 0.0
    
    def _get_latest_features(self, market_data, x_symbols, symbol):
        """获取最新特征数据"""
        try:
            if symbol not in market_data:
                return None
            
            df = market_data[symbol]
            if len(df) == 0:
                return None
            
            latest = df.iloc[-1]
            features = []
            
            # 确保特征生成顺序与训练时一致
            for s in x_symbols:
                if s in market_data:
                    s_df = market_data[s]
                    if len(s_df) > 0:
                        s_latest = s_df.iloc[-1]
                        features.extend([
                            s_latest['returns'] if not pd.isna(s_latest['returns']) else 0,
                            s_latest['rsi'] if not pd.isna(s_latest['rsi']) else 50,
                            s_latest['volume_ratio'] if not pd.isna(s_latest['volume_ratio']) else 1,
                            s_latest['volatility'] if not pd.isna(s_latest['volatility']) else 0,
                            s_latest['macd'] if not pd.isna(s_latest['macd']) else 0,
                            s_latest['price_position'] if not pd.isna(s_latest['price_position']) else 0.5
                        ])
                    else:
                        features.extend([0, 50, 1, 0, 0, 0.5])
                else:
                    features.extend([0, 50, 1, 0, 0, 0.5])
            
            # 验证特征数量
            expected_features = len(x_symbols) * 6  # 每个股票6个特征
            if len(features) != expected_features:
                print(f"⚠️ 特征数量不匹配: 期望 {expected_features}, 实际 {len(features)}")
                return None
            
            # 验证与训练时的特征数量一致
            if self.trained_feature_count > 0 and len(features) != self.trained_feature_count:
                print(f"⚠️ 特征数量与训练时不匹配: 训练时 {self.trained_feature_count}, 当前 {len(features)}")
                return None
            
            return features
            
        except Exception as e:
            print(f"❌ 获取最新特征失败: {str(e)}")
            return None
    
    def _calculate_position_weight(self, symbol, score, cov_matrix, symbols):
        """计算个股权重"""
        try:
            # 基础权重
            base_weight = score
            
            # 考虑相关性惩罚
            if symbol in cov_matrix.index:
                correlations = []
                for other_symbol in symbols:
                    if other_symbol != symbol and other_symbol in cov_matrix.index:
                        corr = cov_matrix.loc[symbol, other_symbol] / (
                            np.sqrt(cov_matrix.loc[symbol, symbol]) * 
                            np.sqrt(cov_matrix.loc[other_symbol, other_symbol])
                        )
                        correlations.append(abs(corr))
                
                if correlations:
                    avg_correlation = np.mean(correlations)
                    # 高相关性惩罚
                    correlation_penalty = 1 - avg_correlation * 0.5
                    base_weight *= correlation_penalty
            
            return max(0, base_weight)
            
        except Exception as e:
            print(f"❌ 计算权重失败: {str(e)}")
            return 0
    
    def _calculate_total_allocation(self, market_state):
        """计算总仓位"""
        allocation_map = {
            'bull': 0.8,      # 牛市
            'neutral': 0.5,   # 中性
            'bear': 0.3       # 熊市
        }
        return allocation_map.get(market_state, 0.5)
    
    def _calculate_risk_metrics(self, positions, cov_matrix):
        """计算风险指标"""
        try:
            if not positions:
                return {}
            
            # 组合波动率
            weights = np.array([pos['weight'] for pos in positions.values()])
            symbols = list(positions.keys())
            
            if len(symbols) > 1 and all(s in cov_matrix.index for s in symbols):
                portfolio_variance = 0
                for i, symbol1 in enumerate(symbols):
                    for j, symbol2 in enumerate(symbols):
                        if symbol1 in cov_matrix.index and symbol2 in cov_matrix.index:
                            portfolio_variance += weights[i] * weights[j] * cov_matrix.loc[symbol1, symbol2]
                
                portfolio_volatility = np.sqrt(portfolio_variance)
            else:
                portfolio_volatility = 0.1  # 默认值
            
            # 最大单股权重
            max_position_weight = max(pos['weight'] for pos in positions.values())
            
            return {
                'portfolio_volatility': portfolio_volatility,
                'max_position_weight': max_position_weight,
                'position_count': len(positions)
            }
            
        except Exception as e:
            print(f"❌ 计算风险指标失败: {str(e)}")
            return {
                'portfolio_volatility': 0.1,
                'max_position_weight': 0.2,
                'position_count': len(positions)
            }
    
    def _empty_recommendations(self):
        """空的投资建议"""
        return {
            'market_state': 'neutral',
            'total_allocation': 0.0,
            'positions': {},
            'risk_metrics': {
                'portfolio_volatility': 0.0,
                'max_position_weight': 0.0,
                'position_count': 0
            }
        }
    
    def backtest(self, market_data, x_symbols, y_symbols, initial_capital=1000000):
        """
        回测功能（参考 finandbacktest 实现）
        
        Args:
            market_data (dict): 市场数据
            x_symbols (list): 因子股票
            y_symbols (list): 目标股票
            initial_capital (float): 初始资金
            
        Returns:
            dict: 回测结果
        """
        print("🔄 开始回测...")
        
        try:
            # 准备回测数据
            backtest_data = self._prepare_backtest_data(market_data, x_symbols, y_symbols)
            
            if backtest_data is None:
                return self._empty_backtest_results(initial_capital)
            
            # 执行回测
            results = self._execute_backtest(backtest_data, initial_capital)
            
            print("✅ 回测完成")
            return results
            
        except Exception as e:
            print(f"❌ 回测失败: {str(e)}")
            return self._empty_backtest_results(initial_capital)
    
    def _prepare_backtest_data(self, market_data, x_symbols, y_symbols):
        """准备回测数据"""
        try:
            # 找到共同的时间范围
            all_dates = []
            for symbol in x_symbols + y_symbols:
                if symbol in market_data:
                    df = market_data[symbol]
                    all_dates.extend(df['date'].tolist())
            
            if not all_dates:
                return None
            
            # 找到最早和最新的日期
            start_date = min(all_dates)
            end_date = max(all_dates)
            
            # 创建时间序列
            date_range = pd.date_range(start=start_date, end=end_date, freq='D')
            date_range = date_range[date_range.weekday < 5]  # 只保留工作日
            
            # 对齐所有数据
            aligned_data = {}
            for symbol in x_symbols + y_symbols:
                if symbol in market_data:
                    df = market_data[symbol].copy()
                    df['date'] = pd.to_datetime(df['date'])
                    df = df.set_index('date')
                    df = df.reindex(date_range, method='ffill')
                    aligned_data[symbol] = df
            
            return {
                'dates': date_range,
                'data': aligned_data,
                'x_symbols': x_symbols,
                'y_symbols': y_symbols
            }
            
        except Exception as e:
            print(f"❌ 准备回测数据失败: {str(e)}")
            return None
    
    def _execute_backtest(self, backtest_data, initial_capital):
        """执行回测"""
        try:
            dates = backtest_data['dates']
            data = backtest_data['data']
            x_symbols = backtest_data['x_symbols']
            y_symbols = backtest_data['y_symbols']
            
            # 初始化
            capital = initial_capital
            positions = {}
            portfolio_values = []
            trades = []
            
            # 每5天调仓一次
            for i in range(20, len(dates), self.config['rebalance_frequency']):
                current_date = dates[i]
                
                # 获取当前市场状态
                market_state = self._get_market_state(data, y_symbols, i)
                
                # 生成投资建议
                recommendations = self._generate_backtest_recommendations(
                    data, x_symbols, y_symbols, i, market_state
                )
                
                # 执行交易
                capital, positions, day_trades = self._execute_trades(
                    capital, positions, recommendations, data, current_date, i
                )
                
                trades.extend(day_trades)
                
                # 记录组合价值
                portfolio_value = self._calculate_portfolio_value(
                    capital, positions, data, current_date, i
                )
                portfolio_values.append({
                    'date': current_date,
                    'value': portfolio_value,
                    'capital': capital
                })
            
            # 计算回测指标
            results = self._calculate_backtest_metrics(
                portfolio_values, initial_capital, trades
            )
            
            return results
            
        except Exception as e:
            print(f"❌ 执行回测失败: {str(e)}")
            return self._empty_backtest_results(initial_capital)
    
    def _get_market_state(self, data, y_symbols, current_idx):
        """获取市场状态"""
        try:
            # 计算大盘20日均线
            market_returns = []
            for symbol in y_symbols:
                if symbol in data and len(data[symbol]) > current_idx:
                    returns = data[symbol].iloc[current_idx]['returns']
                    if not pd.isna(returns):
                        market_returns.append(returns)
            
            if market_returns:
                avg_return = np.mean(market_returns)
                if avg_return > 0.01:  # 1%
                    return 'bull'
                elif avg_return < -0.01:  # -1%
                    return 'bear'
                else:
                    return 'neutral'
            
            return 'neutral'
            
        except Exception as e:
            print(f"❌ 获取市场状态失败: {str(e)}")
            return 'neutral'
    
    def _generate_backtest_recommendations(self, data, x_symbols, y_symbols, current_idx, market_state):
        """生成回测投资建议"""
        try:
            # 模拟模型预测结果
            positions = {}
            total_weight = 0
            
            for symbol in x_symbols:
                if symbol in data and len(data[symbol]) > current_idx:
                    # 简单的技术指标评分
                    df = data[symbol]
                    current = df.iloc[current_idx]
                    
                    # RSI评分
                    rsi_score = 0
                    if not pd.isna(current['rsi']):
                        if current['rsi'] < 30:
                            rsi_score = 0.8  # 超卖
                        elif current['rsi'] > 70:
                            rsi_score = 0.2  # 超买
                        else:
                            rsi_score = 0.5  # 中性
                    else:
                        rsi_score = 0.5
                    
                    # 移动平均线评分
                    ma_score = 0
                    if not pd.isna(current['close']) and not pd.isna(current['ma20']):
                        if current['close'] > current['ma20']:
                            ma_score = 0.7
                        else:
                            ma_score = 0.3
                    else:
                        ma_score = 0.5
                    
                    # 综合评分
                    combined_score = (rsi_score + ma_score) / 2
                    
                    positions[symbol] = {
                        'weight': combined_score,
                        'allocation': 0
                    }
                    total_weight += combined_score
            
            # 归一化权重
            if total_weight > 0:
                for symbol in positions:
                    positions[symbol]['weight'] /= total_weight
            
            # 计算总仓位
            total_allocation = self._calculate_total_allocation(market_state)
            
            # 计算个股权重
            for symbol in positions:
                positions[symbol]['allocation'] = positions[symbol]['weight'] * total_allocation
            
            return {
                'market_state': market_state,
                'total_allocation': total_allocation,
                'positions': positions
            }
            
        except Exception as e:
            print(f"❌ 生成回测建议失败: {str(e)}")
            return self._empty_recommendations()
    
    def _execute_trades(self, capital, positions, recommendations, data, current_date, current_idx):
        """执行交易"""
        try:
            trades = []
            new_positions = {}
            
            # 平仓不在建议中的持仓
            for symbol, position in positions.items():
                if symbol not in recommendations['positions']:
                    if symbol in data and len(data[symbol]) > current_idx:
                        price = data[symbol].iloc[current_idx]['close']
                        if not pd.isna(price) and price > 0:
                            # 卖出
                            sell_value = position['shares'] * price * (1 - self.config['commission'])
                            capital += sell_value
                            trades.append({
                                'date': current_date,
                                'symbol': symbol,
                                'action': 'sell',
                                'shares': position['shares'],
                                'price': price,
                                'value': sell_value
                            })
            
            # 调整持仓
            for symbol, rec in recommendations['positions'].items():
                if symbol in data and len(data[symbol]) > current_idx:
                    price = data[symbol].iloc[current_idx]['close']
                    if not pd.isna(price) and price > 0:
                        target_value = capital * rec['allocation']
                        target_shares = target_value / price
                        
                        current_shares = positions.get(symbol, {}).get('shares', 0)
                        shares_diff = target_shares - current_shares
                        
                        if abs(shares_diff) > 0.01:  # 最小交易单位
                            if shares_diff > 0:  # 买入
                                buy_value = shares_diff * price * (1 + self.config['commission'])
                                if buy_value <= capital:
                                    capital -= buy_value
                                    new_positions[symbol] = {'shares': target_shares}
                                    trades.append({
                                        'date': current_date,
                                        'symbol': symbol,
                                        'action': 'buy',
                                        'shares': shares_diff,
                                        'price': price,
                                        'value': buy_value
                                    })
                            else:  # 卖出
                                sell_value = abs(shares_diff) * price * (1 - self.config['commission'])
                                capital += sell_value
                                new_positions[symbol] = {'shares': target_shares}
                                trades.append({
                                    'date': current_date,
                                    'symbol': symbol,
                                    'action': 'sell',
                                    'shares': abs(shares_diff),
                                    'price': price,
                                    'value': sell_value
                                })
                        else:
                            new_positions[symbol] = {'shares': current_shares}
            
            return capital, new_positions, trades
            
        except Exception as e:
            print(f"❌ 执行交易失败: {str(e)}")
            return capital, positions, []
    
    def _calculate_portfolio_value(self, capital, positions, data, current_date, current_idx):
        """计算组合价值"""
        try:
            portfolio_value = capital
            
            for symbol, position in positions.items():
                if symbol in data and len(data[symbol]) > current_idx:
                    price = data[symbol].iloc[current_idx]['close']
                    if not pd.isna(price) and price > 0:
                        portfolio_value += position['shares'] * price
            
            return portfolio_value
            
        except Exception as e:
            print(f"❌ 计算组合价值失败: {str(e)}")
            return capital
    
    def _calculate_backtest_metrics(self, portfolio_values, initial_capital, trades):
        """计算回测指标"""
        try:
            if not portfolio_values:
                return self._empty_backtest_results(initial_capital)
            
            # 提取价值序列
            values = [pv['value'] for pv in portfolio_values]
            dates = [pv['date'] for pv in portfolio_values]
            
            # 计算收益率
            total_return = (values[-1] - initial_capital) / initial_capital
            
            # 计算最大回撤
            max_drawdown = 0
            peak = values[0]
            for value in values:
                if value > peak:
                    peak = value
                drawdown = (peak - value) / peak
                max_drawdown = max(max_drawdown, drawdown)
            
            # 计算夏普比率（简化版）
            returns = []
            for i in range(1, len(values)):
                daily_return = (values[i] - values[i-1]) / values[i-1]
                returns.append(daily_return)
            
            if returns:
                avg_return = np.mean(returns)
                std_return = np.std(returns)
                sharpe_ratio = avg_return / std_return if std_return > 0 else 0
            else:
                sharpe_ratio = 0
            
            return {
                'initial_capital': initial_capital,
                'final_capital': values[-1],
                'total_return': total_return,
                'max_drawdown': max_drawdown,
                'sharpe_ratio': sharpe_ratio,
                'total_trades': len(trades),
                'portfolio_values': portfolio_values,
                'trades': trades
            }
            
        except Exception as e:
            print(f"❌ 计算回测指标失败: {str(e)}")
            return self._empty_backtest_results(initial_capital)
    
    def _empty_backtest_results(self, initial_capital):
        """空的回测结果"""
        return {
            'initial_capital': initial_capital,
            'final_capital': initial_capital,
            'total_return': 0.0,
            'max_drawdown': 0.0,
            'sharpe_ratio': 0.0,
            'total_trades': 0,
            'portfolio_values': [],
            'trades': []
        }
    
    def optimize_parameters(self, market_data, x_symbols, y_symbols):
        """参数优化（占位符实现）"""
        print("🔧 开始参数优化...")
        
        # 简单的参数优化示例
        optimized_params = {
            'lookback_days': 60,
            'max_positions': 10,
            'max_position_size': 0.2,
            'min_correlation': 0.6
        }
        
        print("✅ 参数优化完成")
        return optimized_params
    
    def get_macro_indicators(self):
        """获取宏观经济指标"""
        print("📈 获取宏观经济指标...")
        
        try:
            macro_data = {}
            
            # GDP数据
            try:
                gdp_data = ak.macro_usa_gdp_monthly()
                if gdp_data is not None and len(gdp_data) > 0:
                    # 找到最新的有效数据（非NaN）
                    latest_valid = gdp_data[gdp_data['今值'].notna()].iloc[-1] if len(gdp_data[gdp_data['今值'].notna()]) > 0 else None
                    macro_data['gdp'] = {
                        'data': gdp_data,
                        'latest_date': latest_valid['日期'] if latest_valid is not None else None,
                        'latest_value': latest_valid['今值'] if latest_valid is not None else None
                    }
                    print("✅ 成功获取GDP数据")
                else:
                    print("❌ GDP数据为空")
            except Exception as e:
                print(f"❌ 获取GDP数据失败: {str(e)}")
            
            # CPI数据
            try:
                cpi_data = ak.macro_usa_cpi_monthly()
                if cpi_data is not None and len(cpi_data) > 0:
                    # 找到最新的有效数据（非NaN）
                    latest_valid = cpi_data[cpi_data['今值'].notna()].iloc[-1] if len(cpi_data[cpi_data['今值'].notna()]) > 0 else None
                    macro_data['cpi'] = {
                        'data': cpi_data,
                        'latest_date': latest_valid['日期'] if latest_valid is not None else None,
                        'latest_value': latest_valid['今值'] if latest_valid is not None else None
                    }
                    print("✅ 成功获取CPI数据")
                else:
                    print("❌ CPI数据为空")
            except Exception as e:
                print(f"❌ 获取CPI数据失败: {str(e)}")
            
            # 利率数据 - 使用美国银行利率
            try:
                interest_data = ak.macro_bank_usa_interest_rate()
                if interest_data is not None and len(interest_data) > 0:
                    # 找到最新的有效数据
                    latest_valid = interest_data.iloc[-1] if len(interest_data) > 0 else None
                    macro_data['interest_rate'] = {
                        'data': interest_data,
                        'latest_date': latest_valid['日期'] if latest_valid is not None else None,
                        'latest_value': latest_valid['今值'] if latest_valid is not None else None
                    }
                    print("✅ 成功获取利率数据")
                else:
                    print("❌ 利率数据为空")
            except Exception as e:
                print(f"❌ 获取利率数据失败: {str(e)}")
            
            return macro_data
            
        except Exception as e:
            print(f"❌ 获取宏观经济指标失败: {str(e)}")
            return {}
    
    def save_model(self, filepath):
        """保存模型"""
        try:
            model_data = {
                'config': self.config,
                'models': self.models,
                'scaler': self.scaler,
                'feature_names': self.feature_names
            }
            
            with open(filepath, 'wb') as f:
                pickle.dump(model_data, f)
            
            print(f"✅ 模型已保存到: {filepath}")
            
        except Exception as e:
            print(f"❌ 保存模型失败: {str(e)}")
    
    def load_model(self, filepath):
        """加载模型"""
        try:
            with open(filepath, 'rb') as f:
                model_data = pickle.load(f)
            
            self.config = model_data['config']
            self.models = model_data['models']
            self.scaler = model_data['scaler']
            self.feature_names = model_data['feature_names']
            
            print(f"✅ 模型已从 {filepath} 加载")
            
        except Exception as e:
            print(f"❌ 加载模型失败: {str(e)}")
    
    def advanced_feature_selection_and_optimization(self, model1_results, market_data, x_symbols, y_symbols):
        """
        高级特征选择和模型优化流程
        
        流程：
        1. 第一个模型计算后，分析特征重要性
        2. 识别影响小的X（预测不准）
        3. 分析相关性，识别表现相同的X
        4. 筛选出对预测有帮助且表现有差异的X
        5. 为模型二及实盘提供优化后的股票池
        
        Args:
            model1_results (dict): 模型1的训练结果
            market_data (dict): 市场数据
            x_symbols (list): 原始X股票列表
            y_symbols (list): Y股票列表
            
        Returns:
            dict: 特征选择和优化结果
        """
        print("\n" + "="*80)
        print("🔍 高级特征选择和模型优化流程")
        print("="*80)
        
        try:
            # 步骤1: 分析特征重要性
            print("\n📊 步骤1: 分析特征重要性...")
            feature_importance_analysis = self._analyze_feature_importance(model1_results, x_symbols)
            
            # 步骤2: 识别影响小的X（预测不准）
            print("\n📊 步骤2: 识别影响小的X...")
            low_impact_stocks = self._identify_low_impact_stocks(feature_importance_analysis, threshold=0.1)
            
            # 步骤3: 相关性分析
            print("\n📊 步骤3: 相关性分析...")
            correlation_analysis = self._advanced_correlation_analysis(market_data, x_symbols, y_symbols)
            
            # 步骤4: 识别高相关性股票（表现相同）
            print("\n📊 步骤4: 识别高相关性股票...")
            high_correlation_groups = self._identify_high_correlation_groups(correlation_analysis, threshold=0.8)
            
            # 步骤5: 综合筛选优化
            print("\n📊 步骤5: 综合筛选优化...")
            optimized_stocks = self._comprehensive_stock_filtering(
                x_symbols, 
                feature_importance_analysis, 
                correlation_analysis, 
                low_impact_stocks, 
                high_correlation_groups
            )
            
            # 步骤6: 生成优化报告
            print("\n📊 步骤6: 生成优化报告...")
            optimization_report = self._generate_optimization_report(
                x_symbols, 
                optimized_stocks, 
                feature_importance_analysis, 
                correlation_analysis, 
                low_impact_stocks, 
                high_correlation_groups
            )
            
            # 保存优化结果
            self.optimized_x_symbols = optimized_stocks
            self.optimization_report = optimization_report
            
            print("\n✅ 高级特征选择和模型优化完成")
            print(f"   原始股票数量: {len(x_symbols)}")
            print(f"   优化后股票数量: {len(optimized_stocks)}")
            print(f"   筛选比例: {len(optimized_stocks)/len(x_symbols)*100:.1f}%")
            
            return {
                'optimized_stocks': optimized_stocks,
                'optimization_report': optimization_report,
                'feature_importance_analysis': feature_importance_analysis,
                'correlation_analysis': correlation_analysis,
                'low_impact_stocks': low_impact_stocks,
                'high_correlation_groups': high_correlation_groups
            }
            
        except Exception as e:
            print(f"❌ 高级特征选择和模型优化失败: {str(e)}")
            return None
    
    def _analyze_feature_importance(self, model1_results, x_symbols):
        """
        分析特征重要性
        
        Args:
            model1_results (dict): 模型1结果
            x_symbols (list): X股票列表
            
        Returns:
            dict: 特征重要性分析结果
        """
        print("   分析特征重要性...")
        
        feature_importance = {}
        
        # 为每个X股票计算综合特征重要性
        for symbol in x_symbols:
            symbol_importance = {
                'total_importance': 0,
                'feature_count': 0,
                'avg_importance': 0,
                'max_importance': 0,
                'min_importance': 0,
                'importance_std': 0,
                'feature_details': {}
            }
            
            # 收集该股票在所有目标模型中的特征重要性
            symbol_features = []
            
            for target_symbol, result in model1_results.items():
                if 'feature_importance' in result:
                    feature_importance_array = result['feature_importance']
                    
                    # 找到该股票对应的特征
                    for i, feature_name in enumerate(self.feature_names):
                        if feature_name.startswith(f"{symbol}_"):
                            if i < len(feature_importance_array):
                                importance_value = feature_importance_array[i]
                                symbol_features.append(importance_value)
                                
                                # 记录特征详情
                                feature_key = feature_name.replace(f"{symbol}_", "")
                                if feature_key not in symbol_importance['feature_details']:
                                    symbol_importance['feature_details'][feature_key] = []
                                symbol_importance['feature_details'][feature_key].append(importance_value)
            
            # 计算统计指标
            if symbol_features:
                symbol_importance['total_importance'] = sum(symbol_features)
                symbol_importance['feature_count'] = len(symbol_features)
                symbol_importance['avg_importance'] = np.mean(symbol_features)
                symbol_importance['max_importance'] = np.max(symbol_features)
                symbol_importance['min_importance'] = np.min(symbol_features)
                symbol_importance['importance_std'] = np.std(symbol_features)
            
            feature_importance[symbol] = symbol_importance
        
        # 按平均重要性排序
        sorted_importance = sorted(
            feature_importance.items(),
            key=lambda x: x[1]['avg_importance'],
            reverse=True
        )
        
        print(f"   特征重要性分析完成，共分析 {len(feature_importance)} 只股票")
        return dict(sorted_importance)
    
    def _identify_low_impact_stocks(self, feature_importance_analysis, threshold=0.1):
        """
        识别影响小的X（预测不准）
        
        Args:
            feature_importance_analysis (dict): 特征重要性分析结果
            threshold (float): 重要性阈值
            
        Returns:
            list: 低影响股票列表
        """
        print(f"   识别低影响股票（阈值: {threshold}）...")
        
        low_impact_stocks = []
        
        for symbol, importance_data in feature_importance_analysis.items():
            avg_importance = importance_data['avg_importance']
            
            if avg_importance < threshold:
                low_impact_stocks.append({
                    'symbol': symbol,
                    'avg_importance': avg_importance,
                    'threshold': threshold,
                    'reason': '特征重要性过低'
                })
        
        # 按重要性排序
        low_impact_stocks.sort(key=lambda x: x['avg_importance'])
        
        print(f"   识别出 {len(low_impact_stocks)} 只低影响股票")
        return low_impact_stocks
    
    def _advanced_correlation_analysis(self, market_data, x_symbols, y_symbols):
        """
        高级相关性分析
        
        Args:
            market_data (dict): 市场数据
            x_symbols (list): X股票列表
            y_symbols (list): Y股票列表
            
        Returns:
            dict: 高级相关性分析结果
        """
        print("   进行高级相关性分析...")
        
        # 计算收益率相关性矩阵
        returns_data = {}
        for symbol in x_symbols + y_symbols:
            if symbol in market_data and market_data[symbol] is not None:
                df = market_data[symbol]
                if 'returns' in df.columns:
                    returns_data[symbol] = df['returns'].dropna()
        
        if len(returns_data) < 2:
            print("   ❌ 数据不足，无法计算相关性")
            return {}
        
        # 创建收益率DataFrame
        returns_df = pd.DataFrame(returns_data)
        returns_df = returns_df.fillna(method='ffill').fillna(0)
        
        # 计算相关性矩阵
        correlation_matrix = returns_df.corr()
        
        # 分析X股票之间的相关性
        x_correlations = {}
        for i, x1 in enumerate(x_symbols):
            if x1 in correlation_matrix.index:
                correlations_with_others = []
                for j, x2 in enumerate(x_symbols):
                    if i != j and x2 in correlation_matrix.index:
                        corr = abs(correlation_matrix.loc[x1, x2])
                        correlations_with_others.append({
                            'symbol': x2,
                            'correlation': corr
                        })
                
                if correlations_with_others:
                    # 按相关性排序
                    correlations_with_others.sort(key=lambda x: x['correlation'], reverse=True)
                    
                    x_correlations[x1] = {
                        'avg_correlation': np.mean([c['correlation'] for c in correlations_with_others]),
                        'max_correlation': max([c['correlation'] for c in correlations_with_others]),
                        'high_correlation_pairs': [c for c in correlations_with_others if c['correlation'] > 0.8],
                        'all_correlations': correlations_with_others
                    }
        
        # 分析X股票与Y股票的相关性
        x_y_correlations = {}
        for x_symbol in x_symbols:
            if x_symbol in correlation_matrix.index:
                y_correlations = []
                for y_symbol in y_symbols:
                    if y_symbol in correlation_matrix.columns:
                        corr = correlation_matrix.loc[x_symbol, y_symbol]
                        y_correlations.append({
                            'symbol': y_symbol,
                            'correlation': corr,
                            'abs_correlation': abs(corr)
                        })
                
                if y_correlations:
                    x_y_correlations[x_symbol] = {
                        'avg_correlation': np.mean([c['abs_correlation'] for c in y_correlations]),
                        'max_correlation': max([c['abs_correlation'] for c in y_correlations]),
                        'correlations': y_correlations
                    }
        
        print(f"   相关性分析完成")
        return {
            'correlation_matrix': correlation_matrix,
            'x_correlations': x_correlations,
            'x_y_correlations': x_y_correlations
        }
    
    def _identify_high_correlation_groups(self, correlation_analysis, threshold=0.8):
        """
        识别高相关性股票组（表现相同）
        
        Args:
            correlation_analysis (dict): 相关性分析结果
            threshold (float): 高相关性阈值
            
        Returns:
            list: 高相关性股票组
        """
        print(f"   识别高相关性股票组（阈值: {threshold}）...")
        
        high_correlation_groups = []
        processed_symbols = set()
        
        if 'x_correlations' not in correlation_analysis:
            return high_correlation_groups
        
        for symbol, corr_data in correlation_analysis['x_correlations'].items():
            if symbol in processed_symbols:
                continue
            
            # 找到与该股票高相关的所有股票
            high_corr_symbols = [symbol]
            processed_symbols.add(symbol)
            
            for pair in corr_data['high_correlation_pairs']:
                if pair['correlation'] >= threshold and pair['symbol'] not in processed_symbols:
                    high_corr_symbols.append(pair['symbol'])
                    processed_symbols.add(pair['symbol'])
            
            # 如果找到高相关性组（至少2只股票）
            if len(high_corr_symbols) >= 2:
                # 计算组内平均相关性
                group_correlations = []
                for i, s1 in enumerate(high_corr_symbols):
                    for j, s2 in enumerate(high_corr_symbols):
                        if i < j and s1 in correlation_analysis['x_correlations']:
                            for pair in correlation_analysis['x_correlations'][s1]['all_correlations']:
                                if pair['symbol'] == s2:
                                    group_correlations.append(pair['correlation'])
                                    break
                
                avg_group_correlation = np.mean(group_correlations) if group_correlations else 0
                
                high_correlation_groups.append({
                    'symbols': high_corr_symbols,
                    'avg_correlation': avg_group_correlation,
                    'size': len(high_corr_symbols),
                    'reason': '高相关性组'
                })
        
        print(f"   识别出 {len(high_correlation_groups)} 个高相关性股票组")
        return high_correlation_groups
    
    def _comprehensive_stock_filtering(self, x_symbols, feature_importance_analysis, 
                                     correlation_analysis, low_impact_stocks, high_correlation_groups):
        """
        综合股票筛选优化
        
        Args:
            x_symbols (list): 原始X股票列表
            feature_importance_analysis (dict): 特征重要性分析
            correlation_analysis (dict): 相关性分析
            low_impact_stocks (list): 低影响股票
            high_correlation_groups (list): 高相关性组
            
        Returns:
            list: 优化后的股票列表
        """
        print("   进行综合股票筛选...")
        
        # 创建股票评分系统
        stock_scores = {}
        
        for symbol in x_symbols:
            score = 0
            reasons = []
            
            # 1. 特征重要性评分 (40%)
            if symbol in feature_importance_analysis:
                importance_data = feature_importance_analysis[symbol]
                importance_score = min(importance_data['avg_importance'] * 40, 40)
                score += importance_score
                reasons.append(f"重要性: {importance_score:.2f}")
            
            # 2. 与Y股票相关性评分 (30%)
            if 'x_y_correlations' in correlation_analysis and symbol in correlation_analysis['x_y_correlations']:
                y_corr_data = correlation_analysis['x_y_correlations'][symbol]
                correlation_score = y_corr_data['avg_correlation'] * 30
                score += correlation_score
                reasons.append(f"相关性: {correlation_score:.2f}")
            
            # 3. 多样性评分 (20%) - 避免高相关性
            diversity_score = 20
            if 'x_correlations' in correlation_analysis and symbol in correlation_analysis['x_correlations']:
                x_corr_data = correlation_analysis['x_correlations'][symbol]
                # 高相关性惩罚
                if x_corr_data['avg_correlation'] > 0.7:
                    diversity_penalty = (x_corr_data['avg_correlation'] - 0.7) * 50
                    diversity_score = max(0, diversity_score - diversity_penalty)
                reasons.append(f"多样性: {diversity_score:.2f}")
            
            # 4. 稳定性评分 (10%) - 基于特征重要性标准差
            stability_score = 10
            if symbol in feature_importance_analysis:
                importance_std = feature_importance_analysis[symbol]['importance_std']
                if importance_std > 0.1:  # 高波动性惩罚
                    stability_penalty = importance_std * 50
                    stability_score = max(0, stability_score - stability_penalty)
                reasons.append(f"稳定性: {stability_score:.2f}")
            
            stock_scores[symbol] = {
                'total_score': score,
                'reasons': reasons,
                'importance_score': feature_importance_analysis.get(symbol, {}).get('avg_importance', 0),
                'correlation_score': correlation_analysis.get('x_y_correlations', {}).get(symbol, {}).get('avg_correlation', 0),
                'diversity_score': diversity_score,
                'stability_score': stability_score
            }
        
        # 排除低影响股票
        low_impact_symbols = {stock['symbol'] for stock in low_impact_stocks}
        filtered_scores = {symbol: data for symbol, data in stock_scores.items() 
                          if symbol not in low_impact_symbols}
        
        # 从高相关性组中选择最佳股票
        for group in high_correlation_groups:
            if len(group['symbols']) > 1:
                # 选择组内评分最高的股票
                group_scores = [(symbol, filtered_scores.get(symbol, {}).get('total_score', 0)) 
                               for symbol in group['symbols'] if symbol in filtered_scores]
                
                if group_scores:
                    # 按评分排序，保留最高分的股票
                    group_scores.sort(key=lambda x: x[1], reverse=True)
                    best_symbol = group_scores[0][0]
                    
                    # 移除组内其他股票
                    for symbol in group['symbols']:
                        if symbol != best_symbol and symbol in filtered_scores:
                            del filtered_scores[symbol]
        
        # 按评分排序并选择前N只股票
        sorted_stocks = sorted(
            filtered_scores.items(),
            key=lambda x: x[1]['total_score'],
            reverse=True
        )
        
        # 选择评分最高的股票（最多保留原始数量的70%）
        max_stocks = min(int(len(x_symbols) * 0.7), len(sorted_stocks))
        optimized_stocks = [stock[0] for stock in sorted_stocks[:max_stocks]]
        
        print(f"   综合筛选完成: {len(optimized_stocks)}/{len(x_symbols)} 只股票")
        return optimized_stocks
    
    def _generate_optimization_report(self, original_stocks, optimized_stocks, 
                                    feature_importance_analysis, correlation_analysis,
                                    low_impact_stocks, high_correlation_groups):
        """
        生成优化报告
        
        Args:
            original_stocks (list): 原始股票列表
            optimized_stocks (list): 优化后股票列表
            feature_importance_analysis (dict): 特征重要性分析
            correlation_analysis (dict): 相关性分析
            low_impact_stocks (list): 低影响股票
            high_correlation_groups (list): 高相关性组
            
        Returns:
            dict: 优化报告
        """
        print("   生成优化报告...")
        
        report = {
            'summary': {
                'original_count': len(original_stocks),
                'optimized_count': len(optimized_stocks),
                'filtered_count': len(original_stocks) - len(optimized_stocks),
                'filtering_ratio': (len(original_stocks) - len(optimized_stocks)) / len(original_stocks) * 100
            },
            'low_impact_stocks': low_impact_stocks,
            'high_correlation_groups': high_correlation_groups,
            'optimized_stocks_details': {},
            'filtering_reasons': {}
        }
        
        # 详细分析优化后的股票
        for symbol in optimized_stocks:
            if symbol in feature_importance_analysis:
                report['optimized_stocks_details'][symbol] = {
                    'importance_rank': list(feature_importance_analysis.keys()).index(symbol) + 1,
                    'avg_importance': feature_importance_analysis[symbol]['avg_importance'],
                    'feature_count': feature_importance_analysis[symbol]['feature_count']
                }
        
        # 分析被过滤股票的原因
        filtered_stocks = set(original_stocks) - set(optimized_stocks)
        for symbol in filtered_stocks:
            reasons = []
            
            # 检查是否为低影响股票
            if any(stock['symbol'] == symbol for stock in low_impact_stocks):
                reasons.append('特征重要性过低')
            
            # 检查是否为高相关性组中被移除的股票
            for group in high_correlation_groups:
                if symbol in group['symbols']:
                    # 检查是否是该组中评分最高的
                    group_scores = [(s, feature_importance_analysis.get(s, {}).get('avg_importance', 0)) 
                                   for s in group['symbols']]
                    group_scores.sort(key=lambda x: x[1], reverse=True)
                    if group_scores and group_scores[0][0] != symbol:
                        reasons.append(f'高相关性组中被替代 (组内排名: {[s[0] for s in group_scores].index(symbol) + 1}/{len(group_scores)})')
                    break
            
            if not reasons:
                reasons.append('综合评分较低')
            
            report['filtering_reasons'][symbol] = reasons
        
        return report
    
    def print_optimization_report(self):
        """
        打印优化报告
        """
        if not hasattr(self, 'optimization_report'):
            print("❌ 没有可用的优化报告")
            return
        
        report = self.optimization_report
        
        print("\n" + "="*80)
        print("📊 特征选择和模型优化报告")
        print("="*80)
        
        # 总体摘要
        summary = report['summary']
        print(f"\n🎯 总体摘要:")
        print(f"   原始股票数量: {summary['original_count']}")
        print(f"   优化后股票数量: {summary['optimized_count']}")
        print(f"   过滤股票数量: {summary['filtered_count']}")
        print(f"   筛选比例: {summary['filtering_ratio']:.1f}%")
        
        # 优化后的股票列表
        if hasattr(self, 'optimized_x_symbols') and self.optimized_x_symbols:
            print(f"\n✅ 优化后的股票列表 (按重要性排序):")
            for i, symbol in enumerate(self.optimized_x_symbols, 1):
                details = report['optimized_stocks_details'].get(symbol, {})
                importance_rank = details.get('importance_rank', 'N/A')
                avg_importance = details.get('avg_importance', 0)
                print(f"   {i:2d}. {symbol} (重要性排名: {importance_rank}, 平均重要性: {avg_importance:.4f})")
        
        # 低影响股票
        if report['low_impact_stocks']:
            print(f"\n⚠️ 低影响股票 (特征重要性过低):")
            for stock in report['low_impact_stocks']:
                print(f"   {stock['symbol']}: 平均重要性 {stock['avg_importance']:.4f} < {stock['threshold']}")
        
        # 高相关性组
        if report['high_correlation_groups']:
            print(f"\n🔗 高相关性股票组:")
            for i, group in enumerate(report['high_correlation_groups'], 1):
                print(f"   组 {i}: {', '.join(group['symbols'])} (平均相关性: {group['avg_correlation']:.3f})")
        
        # 被过滤股票的原因
        if report['filtering_reasons']:
            print(f"\n🗑️ 被过滤股票及原因:")
            for symbol, reasons in report['filtering_reasons'].items():
                print(f"   {symbol}: {', '.join(reasons)}")
        
        print("="*80)
    
    def get_optimized_stocks_for_model2_and_live_trading(self):
        """
        获取用于模型二和实盘交易的优化股票列表
        
        Returns:
            list: 优化后的股票列表
        """
        if hasattr(self, 'optimized_x_symbols') and self.optimized_x_symbols:
            print(f"✅ 使用优化后的股票列表进行模型二和实盘交易: {len(self.optimized_x_symbols)} 只股票")
            return self.optimized_x_symbols
        else:
            print("⚠️ 没有优化后的股票列表，使用原始股票列表")
            return self.x_symbols if hasattr(self, 'x_symbols') else []


# 使用示例
if __name__ == "__main__":
    # 初始化模型
    prophecy = ProphecyModel()
    
    # 定义股票池
    x_symbols = ["AAPL.US", "MSFT.US", "GOOGL.US", "AMZN.US", "TSLA.US", 
                 "META.US", "NVDA.US", "NFLX.US", "ADBE.US", "CRM.US",
                 "ORCL.US", "INTC.US", "AMD.US", "QCOM.US", "AVGO.US"]  # 因子股票
    y_symbols = ["SPY.US", "QQQ.US", "DIA.US"]  # 目标股票（大盘）
    
    print("🚀 ProphecyModel 高级特征选择和模型优化演示")
    print("="*80)
    
    # 获取数据
    print("\n📊 步骤1: 获取市场数据...")
    market_data = prophecy.get_market_data(x_symbols + y_symbols)
    
    if len(market_data) > 0:
        print(f"✅ 成功获取 {len(market_data)} 只股票的数据")
        
        # 准备特征数据
        print("\n📊 步骤2: 准备特征数据...")
        X_train, X_test, y_train, y_test = prophecy.prepare_features(
            market_data, x_symbols, y_symbols
        )
        
        if X_train is not None:
            print(f"✅ 特征数据准备完成")
            print(f"   训练集: {len(X_train)} 条记录")
            print(f"   测试集: {len(X_test)} 条记录")
            print(f"   特征数量: {len(X_train.columns)}")
            print(f"   目标数量: {len(y_train.columns)}")
            
            # 训练模型1（启用高级特征选择和优化）
            print("\n📊 步骤3: 训练模型1（启用高级特征选择和优化）...")
            model1_results = prophecy.train_model_1(
                X_train, y_train, 
                model_name='rf', 
                auto_filter=False,  # 关闭基础筛选
                enable_advanced_optimization=True  # 启用高级优化
            )
            
            if model1_results:
                print(f"✅ 模型1训练完成，共训练 {len(model1_results)} 个目标模型")
                
                # 获取优化后的股票列表
                optimized_stocks = prophecy.get_optimized_stocks_for_model2_and_live_trading()
                
                if optimized_stocks:
                    print(f"\n📊 步骤4: 使用优化后的股票训练模型2...")
                    print(f"   优化后股票: {', '.join(optimized_stocks)}")
                    
                    # 训练模型2（使用优化后的股票）
                    model2_results = {}
                    for symbol in optimized_stocks:
                        if symbol in market_data:
                            print(f"   训练 {symbol} 的个股模型...")
                            result = prophecy.train_model_2(market_data, symbol)
                            if result:
                                model2_results[symbol] = result
                    
                    print(f"✅ 模型2训练完成，共训练 {len(model2_results)} 个个股模型")
                    
                    # 生成投资建议（使用优化后的股票）
                    print(f"\n📊 步骤5: 生成投资组合建议...")
                    recommendations = prophecy.generate_portfolio_recommendations(
                        model1_results, model2_results, market_data, x_symbols, y_symbols,
                        current_market_state='neutral',
                        use_optimized_stocks=True  # 使用优化后的股票
                    )
                    
                    if recommendations and recommendations['portfolio']:
                        print(f"\n📋 投资组合建议:")
                        print(f"   优化来源: {recommendations['optimization_info']['optimization_source']}")
                        print(f"   总仓位: {recommendations['total_allocation']:.1%}")
                        print(f"   股票数量: {len(recommendations['portfolio'])}")
                        print(f"   优化比例: {recommendations['optimization_info']['optimization_ratio']:.1f}%")
                        
                        print(f"\n📊 个股配置:")
                        for symbol, pos in recommendations['portfolio'].items():
                            print(f"   {symbol}: {pos['weight']:.1%} (评分: {pos['score']:.3f}, 优化评分: {pos['optimization_score']:.3f})")
                        
                        # 风险指标
                        if recommendations['risk_metrics']:
                            risk = recommendations['risk_metrics']
                            print(f"\n⚠️ 风险指标:")
                            print(f"   组合波动率: {risk.get('portfolio_volatility', 0):.3f}")
                            print(f"   最大单股权重: {risk.get('max_position_weight', 0):.1%}")
                            print(f"   持仓数量: {risk.get('position_count', 0)}")
                    else:
                        print("❌ 无法生成投资组合建议")
                else:
                    print("❌ 没有优化后的股票列表")
            else:
                print("❌ 模型1训练失败")
        else:
            print("❌ 特征数据准备失败")
    else:
        print("❌ 无法获取市场数据")
    
    print("\n" + "="*80)
    print("✅ ProphecyModel 高级特征选择和模型优化演示完成")
    print("="*80)
